{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import re\n",
    "import json\n",
    "\n",
    "\n",
    "# 待处理的法律文本\n",
    "def parse_law(text,title):\n",
    "\n",
    "    # 使用正则表达式按照章的标识符分割文本\n",
    "    chapters = re.split(r'(第[一二三四五六七八九十]+章)', text)\n",
    "\n",
    "    current_chapter = \"\"\n",
    "    for chunk in chapters:\n",
    "        # 去除空白字符后，检查当前片段是否为章节标题\n",
    "        if re.match(r'^第[一二三四五六七八九十]+章', chunk := chunk.strip()):\n",
    "            current_chapter = chunk  # 更新当前章标题\n",
    "        else:\n",
    "            # 查找所有的条目以及对应的内容，确保忽略换行符\n",
    "            articles = re.findall(\n",
    "                r'(第[一二三四五六七八九十百]+条)\\s+(.*?)(?=(第[一二三四五六七八九十百]+条)|$)',\n",
    "                chunk, re.DOTALL\n",
    "            )\n",
    "            # 遍历条目，构建返回结果\n",
    "            for article_num, content, _ in articles:\n",
    "                # 移除内容中的所有换行符，并生成结构化的结果\n",
    "                clean_content = re.sub(r'\\s+', '', content.strip())\n",
    "\n",
    "                yield {\n",
    "                    \"instruction\":\"请根据\"+\"《\" +title+\"》\" + current_chapter + article_num+\"回答下列问题\",\n",
    "                    \"input\": \"\",\n",
    "                    \"output\": clean_content\n",
    "                }\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 将法律内容拆分为条文，并转换成alpaca格式的数据集\n",
    "def law_to_alpaca_format(file_path):\n",
    "    df = pd.read_excel(file_path)\n",
    "    title = df['title']\n",
    "    content = df['content']\n",
    "    processed_law = []\n",
    "    for row in range(len(df)):\n",
    "        index = df.index[row]\n",
    "        current_law = list(parse_law(content[index],title[index]))\n",
    "        processed_law = processed_law + current_law\n",
    "\n",
    "    # 输出JSON格式的结构化结果到文件\n",
    "    with open('output.json', 'w', encoding='utf-8') as out_file:\n",
    "        json.dump(processed_law, out_file, ensure_ascii=False, indent=2)\n",
    "\n",
    "    print(\"处理完成，数据已保存到 output.json 文件中。\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "处理完成，数据已保存到 output.json 文件中。\n"
     ]
    }
   ],
   "source": [
    "law_to_alpaca_format(\"predata.xlsx\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "已生成 splitted_data\\chinese_law_1.json，包含 450 条数据\n",
      "已生成 splitted_data\\chinese_law_2.json，包含 450 条数据\n",
      "已生成 splitted_data\\chinese_law_3.json，包含 450 条数据\n",
      "已生成 splitted_data\\chinese_law_4.json，包含 450 条数据\n",
      "已生成 splitted_data\\chinese_law_5.json，包含 450 条数据\n",
      "已生成 splitted_data\\chinese_law_6.json，包含 450 条数据\n",
      "已生成 splitted_data\\chinese_law_7.json，包含 450 条数据\n",
      "已生成 splitted_data\\chinese_law_8.json，包含 450 条数据\n",
      "已生成 splitted_data\\chinese_law_9.json，包含 450 条数据\n",
      "已生成 splitted_data\\chinese_law_10.json，包含 456 条数据\n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "import pandas as pd\n",
    "import os\n",
    "\n",
    "def split_data(file_path, output_dir=\"splitted_data\", num_splits=10):\n",
    "    \"\"\"将 JSON 或 CSV 文件拆分成 num_splits 份\"\"\"\n",
    "    os.makedirs(output_dir, exist_ok=True)\n",
    "\n",
    "    # 读取数据\n",
    "    if file_path.endswith(\".json\"):\n",
    "        with open(file_path, \"r\", encoding=\"utf-8\") as f:\n",
    "            data = json.load(f)\n",
    "    elif file_path.endswith(\".csv\"):\n",
    "        data = pd.read_csv(file_path)\n",
    "    else:\n",
    "        raise ValueError(\"仅支持 JSON 和 CSV 文件\")\n",
    "\n",
    "    # 计算每份数据的大小\n",
    "    total_size = len(data)\n",
    "    chunk_size = total_size // num_splits\n",
    "\n",
    "    # 拆分数据\n",
    "    for i in range(num_splits):\n",
    "        start_idx = i * chunk_size\n",
    "        end_idx = (i + 1) * chunk_size if i < num_splits - 1 else total_size\n",
    "        part_data = data[start_idx:end_idx]\n",
    "\n",
    "        # 生成文件名\n",
    "        output_file = os.path.join(output_dir, f\"chinese_law_{i+1}.{file_path.split('.')[-1]}\")\n",
    "\n",
    "        # 保存数据\n",
    "        if file_path.endswith(\".json\"):\n",
    "            with open(output_file, \"w\", encoding=\"utf-8\") as f:\n",
    "                json.dump(part_data, f, ensure_ascii=False, indent=2)\n",
    "        elif file_path.endswith(\".csv\"):\n",
    "            part_data.to_csv(output_file, index=False)\n",
    "\n",
    "        print(f\"已生成 {output_file}，包含 {len(part_data)} 条数据\")\n",
    "\n",
    "# 运行示例\n",
    "split_data(\"chinese_law.json\")  # 替换为你的数据文件名\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'主体': [], '行为动词': ['', '', ''], '法律后果': [], '条件状语': []}\n"
     ]
    }
   ],
   "source": [
    "import spacy\n",
    "\n",
    "nlp = spacy.load(\"zh_core_web_trf\")\n",
    "\n",
    "def parse_legal_clause(text):\n",
    "    doc = nlp(text)\n",
    "    elements = {\n",
    "        \"主体\": [ent.text for ent in doc.ents if ent.label_ in [\"ORG\", \"PERSON\"]],\n",
    "        \"行为动词\": [token.lemma_ for token in doc if token.pos_ == \"VERB\"],\n",
    "        \"法律后果\": [token.text for token in doc if token.text in [\"处罚\", \"解除劳动合同\", \"赔偿\", \"承担责任\"]],\n",
    "        \"条件状语\": [token.text for token in doc if token.dep_ == \"advcl\"]\n",
    "    }\n",
    "    return elements\n",
    "\n",
    "# 示例条款：《劳动合同法》第三十八条 \n",
    "clause = \"用人单位未及时足额支付劳动报酬的，劳动者可以解除劳动合同\"\n",
    "print(parse_legal_clause(clause))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'主体': [], '行为动词': ['', '', ''], '法律后果': [], '条件状语': []}\n"
     ]
    }
   ],
   "source": [
    "import spacy\n",
    "from spacy.matcher import Matcher\n",
    "\n",
    "nlp = spacy.load(\"zh_core_web_trf\")\n",
    "\n",
    "# 定义法律后果关键词\n",
    "legal_consequences = [\"解除劳动合同\", \"赔偿\", \"承担责任\", \"罚款\", \"拘留\", \"追究刑事责任\"]\n",
    "\n",
    "# 初始化 Matcher\n",
    "matcher = Matcher(nlp.vocab)\n",
    "for consequence in legal_consequences:\n",
    "    pattern = [{\"TEXT\": consequence}]\n",
    "    matcher.add(consequence, [pattern])\n",
    "\n",
    "def parse_legal_clause(text):\n",
    "    doc = nlp(text)\n",
    "    \n",
    "    # 提取主体（组织、个人）\n",
    "    subjects = [ent.text for ent in doc.ents if ent.label_ in [\"ORG\", \"PERSON\"]]\n",
    "    \n",
    "    # 提取核心行为动词\n",
    "    verbs = [token.lemma_ for token in doc if token.pos_ == \"VERB\" or token.dep_ == \"ROOT\"]\n",
    "    \n",
    "    # 使用规则匹配查找法律后果\n",
    "    matches = matcher(doc)\n",
    "    consequences = [doc[start:end].text for match_id, start, end in matches]\n",
    "    \n",
    "    # 提取条件状语（状语从句）\n",
    "    adverbials = [token.text for token in doc if token.dep_ == \"advcl\"]\n",
    "\n",
    "    elements = {\n",
    "        \"主体\": subjects,\n",
    "        \"行为动词\": verbs,\n",
    "        \"法律后果\": consequences,\n",
    "        \"条件状语\": adverbials\n",
    "    }\n",
    "    \n",
    "    return elements\n",
    "\n",
    "# 测试案例：《劳动合同法》第三十八条 \n",
    "clause = \"用人单位未及时足额支付劳动报酬的，劳动者可以解除劳动合同\"\n",
    "print(parse_legal_clause(clause))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import spacy\n",
    "from transformers import pipeline\n",
    "\n",
    "# 加载spaCy的中文模型\n",
    "nlp = spacy.load(\"zh_core_web_sm\")\n",
    "\n",
    "# 加载transformers的问答生成模型\n",
    "question_generator = pipeline(\"text2text-generation\", model=\"mrm8488/t5-base-finetuned-question-generation-ap\")\n",
    "\n",
    "# 从Excel文件中读取法律条款\n",
    "def load_legal_articles_from_excel(file_path, sheet_name=\"条款\", column_name=\"条款内容\"):\n",
    "    try:\n",
    "        # 使用pandas读取Excel文件\n",
    "        df = pd.read_excel(file_path, sheet_name=sheet_name)\n",
    "        if column_name not in df.columns:\n",
    "            raise ValueError(f\"列名 '{column_name}' 不存在于Excel文件中！\")\n",
    "        \n",
    "        # 提取条款内容\n",
    "        legal_articles = df[column_name].dropna().tolist()  # 去除空值并转换为列表\n",
    "        return legal_articles\n",
    "    except Exception as e:\n",
    "        print(f\"读取Excel文件时出错: {e}\")\n",
    "        return []\n",
    "\n",
    "# 使用spaCy提取关键信息\n",
    "def extract_keywords(article):\n",
    "    doc = nlp(article)\n",
    "    keywords = {\n",
    "        \"subject\": [],  # 主体\n",
    "        \"action\": [],   # 行为\n",
    "        \"condition\": [],  # 条件\n",
    "        \"responsibility\": []  # 责任\n",
    "    }\n",
    "    \n",
    "    for token in doc:\n",
    "        if token.pos_ == \"NOUN\" or token.pos_ == \"PROPN\":  # 名词或专有名词\n",
    "            keywords[\"subject\"].append(token.text)\n",
    "        if token.pos_ == \"VERB\":  # 动词\n",
    "            keywords[\"action\"].append(token.text)\n",
    "        if token.dep_ == \"advmod\" or token.dep_ == \"acl\":  # 条件修饰\n",
    "            keywords[\"condition\"].append(token.text)\n",
    "        if token.dep_ == \"dobj\" or token.dep_ == \"pobj\":  # 责任对象\n",
    "            keywords[\"responsibility\"].append(token.text)\n",
    "    \n",
    "    return {k: list(set(v)) for k, v in keywords.items()}  # 去重\n",
    "\n",
    "# 使用transformers生成问题\n",
    "def generate_questions_with_transformers(article):\n",
    "    questions = question_generator(f\"generate question: {article}\")\n",
    "    return [q['generated_text'] for q in questions]\n",
    "\n",
    "# 批量生成问题\n",
    "def batch_generate_questions(articles):\n",
    "    all_questions = []\n",
    "    for article in articles:\n",
    "        # 提取关键词\n",
    "        keywords = extract_keywords(article)\n",
    "        print(f\"Extracted Keywords: {keywords}\")\n",
    "        \n",
    "        # 使用transformers生成问题\n",
    "        generated_questions = generate_questions_with_transformers(article)\n",
    "        all_questions.extend(generated_questions)\n",
    "    \n",
    "    return all_questions\n",
    "\n",
    "# 主程序\n",
    "if __name__ == \"__main__\":\n",
    "    # Excel文件路径\n",
    "    excel_file_path = \"output_texts.xlsx\"\n",
    "    \n",
    "    # 从Excel文件中加载法律条款\n",
    "    legal_articles = load_legal_articles_from_excel(excel_file_path)\n",
    "    \n",
    "    if not legal_articles:\n",
    "        print(\"未找到任何法律条款，请检查Excel文件内容！\")\n",
    "    else:\n",
    "        # 批量生成问题\n",
    "        questions = batch_generate_questions(legal_articles)\n",
    "        \n",
    "        # 输出生成的问题\n",
    "        print(\"\\nGenerated Questions:\")\n",
    "        for i, q in enumerate(questions, 1):\n",
    "            print(f\"{i}. {q}\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "data_pre",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
